package com.cgy.utils.spider.downloader;

import java.io.IOException;
import java.net.MalformedURLException;

import com.gargoylesoftware.htmlunit.*;

import lombok.Getter;
import lombok.Setter;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.Task;
import us.codecraft.webmagic.downloader.AbstractDownloader;
import us.codecraft.webmagic.selector.Html;
import us.codecraft.webmagic.selector.PlainText;
/**
 * HtmlUnit下载器
 * @author CH
 *
 */
@Getter
@Setter
public class HtmlUnitDownloader extends AbstractDownloader{

	private WebClient webClient;
	private int poolSize = 1;
	
	{
		webClient = new WebClient(BrowserVersion.CHROME);
		WebClientOptions options = webClient.getOptions();
		options.setJavaScriptEnabled(true);
		options.setUseInsecureSSL(true);
		options.setCssEnabled(false);
		options.setThrowExceptionOnScriptError(false);
		options.setThrowExceptionOnFailingStatusCode(false);
		options.setPrintContentOnFailingStatusCode(false);
		options.setRedirectEnabled(true);
		options.setAppletEnabled(false);
		options.setPopupBlockerEnabled(true);
		webClient.setAjaxController(new NicelyResynchronizingAjaxController());
		webClient.waitForBackgroundJavaScript(1000);
	}
	
	@SuppressWarnings("deprecation")
	@Override
	public Page download(Request request, Task task) {
		Page page = new Page();
		try {
			com.gargoylesoftware.htmlunit.Page htmlPage = webClient.getPage(request.getUrl());
			String textContent = htmlPage.getWebResponse().getContentAsString();
			page.setRawText(textContent);
			page.setUrl(new PlainText(request.getUrl()));
			page.setHtml(new Html(textContent, request.getUrl()));
			page.setRequest(request);
			
		} catch (FailingHttpStatusCodeException e) {
			e.printStackTrace();
		} catch (MalformedURLException e) {
			e.printStackTrace();
		} catch (IOException e) {
			e.printStackTrace();
		}
		return page;
	}

	@Override
	public void setThread(int num) {
		this.poolSize = num;
	}


}
